{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "c5bf3ae2",
   "metadata": {},
   "source": [
    "# CBOW模型示例\n",
    "\n",
    "本notebook演示如何使用CBOW(Continuous Bag of Words)模型进行词向量训练和推理。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "121512ac",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1, 3, 4, 5, 1, 6]\n",
      "[1, 7, 8, 2, 1, 9]\n",
      "[1, 10, 11, 2, 1, 12]\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense, Embedding, Lambda\n",
    "from tensorflow.keras.preprocessing.text import Tokenizer\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.decomposition import PCA\n",
    "\n",
    "corpus = [\n",
    "    'The cat sat on the mat',\n",
    "    'The dog ran in the park',\n",
    "    'The bird sang in the tree'\n",
    "]\n",
    "\n",
    "# 分词\n",
    "tokenizer = Tokenizer()\n",
    "tokenizer.fit_on_texts(corpus)\n",
    "# 每一个文档转换成一个整形序列，整形值其实是该词的构造出的词典的索引位置，该词典不重要\n",
    "sequences = tokenizer.texts_to_sequences(corpus)\n",
    "for ele in sequences:\n",
    "    print(ele)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c7bfdfde",
   "metadata": {},
   "source": [
    "上一步构造出来的也是向量，但如果词库很大（现实词库确实很大），那向量的维度太大了。\n",
    "\n",
    "下一步，开始构造CBOW的target和target上下文词。以上面第一条句子为例，context-target 是类似 `[[The, sat], cat]` 这样的结构（如果window_size=1）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "c7540a32",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1, 3, 5, 1], 4\n",
      "[3, 4, 1, 6], 5\n",
      "[1, 7, 2, 1], 8\n",
      "[7, 8, 1, 9], 2\n",
      "[1, 10, 2, 1], 11\n",
      "[10, 11, 1, 12], 2\n"
     ]
    }
   ],
   "source": [
    "window_size = 2\n",
    "embedding_size = 10 # 词向量维度\n",
    "vocab_size = len(tokenizer.word_index) + 1 # 词典大小\n",
    "\n",
    "# 生成target和该target的上下文\n",
    "contexts = []\n",
    "targets = []\n",
    "for sequence in sequences:\n",
    "    for i in range(window_size, len(sequence) - window_size):\n",
    "        target = sequence[i]\n",
    "        context = sequence[i - window_size:i] + sequence[i + 1:i + window_size + 1]\n",
    "        contexts.append(context)\n",
    "        targets.append(target)\n",
    "\n",
    "for i in range(len(targets)):\n",
    "    print(f'{contexts[i]}, {targets[i]}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "cc318b55",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<keras.src.callbacks.history.History at 0x773aae73f4d0>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 定义CBOW模型\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Embedding(input_dim=vocab_size, output_dim=embedding_size, input_length=2 * window_size))\n",
    "model.add(Lambda(lambda x: tf.reduce_mean(x, axis=1)))\n",
    "model.add(Dense(units=vocab_size, activation='softmax'))\n",
    "model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "\n",
    "# 训练数据\n",
    "X = np.array(contexts)\n",
    "y = to_categorical(targets, num_classes=vocab_size)\n",
    "model.fit(X, y, epochs=100, verbose=0)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
